import { Func } from "../../common/types";
import { Diagnostic, DiagnosticMessage, errorMessages } from "../diagnostic";
import {
    Comment,
    DelcareKeywordKind,
    DmsSectionKeywordKind,
    FileKind,
    KeywordTokenKind,
    MDMKeywordKind,
    Scanner,
    ScannerState,
    ScannerContextType,
    SyntaxKind,
    TextPosition,
    TokenFlags
} from "./types";
import {
    binarySearch,
    compareNumberValue,
    stringToDeclareKeyword,
    stringToDmsSectionKeyword,
    stringToMrsKeyword,
    stringToMetadataKeyword,
    stringToPreprocessorKeyword
} from "./util";

export function isIdentifierOrReservedWord(flag: SyntaxKind): boolean {
    return flag === SyntaxKind.identifier
        || (flag >= SyntaxKind.firstKeyword && flag <= SyntaxKind.lastKeyword)
        || (flag >= SyntaxKind.firstSectionKeyword && flag <= SyntaxKind.lastSectionKeyword)
        || (flag >= SyntaxKind.firstMDMKeyword && flag <= SyntaxKind.lastMDMKeyword)
        || (flag >= SyntaxKind.firstDeclareKeyword && flag <= SyntaxKind.lastDeclareKeyword);
}

export function isReservedWord(flag: SyntaxKind, fileFlag: FileKind): boolean {
    let result = flag >= SyntaxKind.firstKeyword && flag <= SyntaxKind.lastKeyword;
    if (!result && fileFlag === FileKind.declare) {
        result = flag >= SyntaxKind.firstDeclareKeyword && flag <= SyntaxKind.lastDeclareKeyword;
    }
    if (!result && fileFlag === FileKind.mdd) {
        result = flag >= SyntaxKind.firstMDMKeyword && flag <= SyntaxKind.lastMDMKeyword;
    }
    return result;
}

export function createScanner(initialText?: string, start?: number, length?: number): Scanner {

    let state: ScannerState;
    let _onError: Func<Diagnostic, void> | undefined;
    let comments: Comment[] = [];

    setText(initialText, start, length);

    return {
        setContext: type => state.context = type,
        saveState,
        recoveryState: (savedState: ScannerState) => state = savedState,
        atBegining: () => state.atFirstLine && state.text.length > state.tokenStart && state.text.substring(0, state.tokenStart).trim() === "",
        atFirstLine: () => state.atFirstLine,
        getStartPos: () => state.startPos,
        getTextPos: () => state.pos,
        getToken: () => state.token,
        getTokenStart: () => state.tokenStart,
        getTokenText: (start?: number, end?: number) => state.text.substring(start !== undefined ? start : state.tokenStart, end !== undefined ? end : state.pos),
        getTokenValue: () => state.tokenValue,
        getTokenFlag: () => state.tokenFlag,
        getComments: () => comments,
        isIdentifier: () => state.token === SyntaxKind.identifier,
        isPreprocessorOperator: () => state.token >= SyntaxKind.firstPPOperator && state.token <= SyntaxKind.lastPPOperator,
        isReservedWord: () => state.token >= SyntaxKind.paper && state.token <= SyntaxKind.sectionKeyword,
        isDelcareReservedWord,
        isTrueLiteral: () => state.token === SyntaxKind.trueKeyword,
        isFalseLiteral: () => state.token === SyntaxKind.falseKeyword,
        isNullLiteral: () => state.token === SyntaxKind.nullKeyword,
        hasPrecedingLinebreak,
        setText,
        onError,
        scan,
        scanRange,
        lookAhead,
        tryScan,
        scanToLinefeedAsString,
        reScanInvalidIdentifier,
    };

    // 跳过'\r\n'或'\r'两种换行
    function skipSingleLineFeed() {
        let ch = state.text.charCodeAt(state.pos);
        let ahead = lookAheadChar();
        if (ch === CharacterCodes.carriageReturn && ahead === CharacterCodes.lineFeed) {
            state.pos += 2;
        }
        else if (ch === CharacterCodes.lineFeed && !isLineBreak(ahead)) {
            state.pos++;
        }
    }

    function skipWhitespace() {
        let ch = currentChar();
        while (isWhiteSpaceSingleLine(ch) && state.pos < state.end) {
            ch = nextChar();
        }
    }

    function skipEmpty() {
        let ch = state.text.charCodeAt(state.pos);
        while (isWhiteSpaceLike(ch) && state.pos < state.end) {
            if (isLineBreak(ch)) {
                state.tokenFlag |= TokenFlags.precedingLinebreak;
                if (state.pos > 0) {
                    state.atFirstLine = false;
                }
            }
            state.pos++;
            ch = state.text.charCodeAt(state.pos);
        }
    }

    function scanSingleLineComment(isPreproComment: boolean) {
        const start = state.pos;
        if (isPreproComment) {
            // 跳过 "//"
            state.pos += 2;
        }
        else {
            // 跳过单引号"'"
            state.pos++;
        }
        const textStart = state.pos;
        let ch = state.text.charCodeAt(state.pos);
        while (true) {
            if (state.pos >= state.end || isLineBreak(ch)) {
                break;
            }
            state.pos++;
            ch = state.text.charCodeAt(state.pos);
        }
        comments.push({
            start: start,
            end: state.pos,
            comments: [ state.text.substring(textStart, state.pos) ]
        });
    }

    function scanMultiLineComment(isPreproComment: boolean) {
        const start = state.pos;
        state.pos += 2;
        let textStart = state.pos;
        const commentTexts: string[] = [];
        while (true) {
            let ch = state.text.charCodeAt(state.pos);
            let nextCh = state.text.charCodeAt(state.pos + 1);
            if (isLineBreak(ch)) {
                commentTexts.push(state.text.substring(textStart, state.pos));
                state.pos++;
                textStart = state.pos;
            }
            if ((!isPreproComment && ch === CharacterCodes.exclamation && nextCh === CharacterCodes.apostrophe) ||
                (isPreproComment && ch === CharacterCodes.asterisk && nextCh === CharacterCodes.slash) || // */
                state.pos >= state.end
            ) {
                let lastLine = state.text.substring(textStart, state.pos);
                if (lastLine.length > 0) {
                    commentTexts.push(lastLine);
                }
                state.pos += 2;
                break;
            }
            state.pos++;
        }
        comments.push({
            start: start,
            end: state.pos,
            comments: commentTexts
        });
    }

    function skipTrivia() {
        skipEmpty();
        comments = [];
        let ch = currentChar();
        let isPreproComment = state.context === ScannerContextType.preprocessor && ch === CharacterCodes.slash;
        // comment
        while (ch === CharacterCodes.apostrophe || isPreproComment) {
            let nextCh = state.text.charCodeAt(state.pos + 1);
            if (nextCh === CharacterCodes.exclamation || (isPreproComment && nextCh === CharacterCodes.asterisk)) {
                scanMultiLineComment(isPreproComment);
            }
            else {
                scanSingleLineComment(isPreproComment);
            }
            skipEmpty();
            ch = currentChar();
        }
    }

    function isDelcareReservedWord(): boolean {
        return state.token >= SyntaxKind.firstDeclareKeyword && state.token <= SyntaxKind.lastDeclareKeyword;
    }

    function saveState(): ScannerState {
        return Object.assign({}, state);
    }

    function setText(newText: string | undefined, start: number | undefined, length: number | undefined): void {
        setState(newText || "", start || 0);
        state.end = length === undefined ? state.text.length : (start || 0) + length;
    }

    function setState(text: string, textPos: number): void {
        state = {
            text: text,
            pos: textPos,
            end: textPos,
            startPos: textPos,
            tokenStart: textPos,
            token: SyntaxKind.unknown,
            tokenValue: "",
            tokenFlag: TokenFlags.none,
            atFirstLine: true,
            context: ScannerContextType.normal,
        };
    }

    function onError(onErrorCallback: Func<Diagnostic, void> | undefined): void {
        _onError = onErrorCallback;
    }

    function hasPrecedingLinebreak(): boolean {
        return (state.tokenFlag & TokenFlags.precedingLinebreak) > 0;
    }

    function error(message: DiagnosticMessage): void;
    function error(message: DiagnosticMessage, start: number, length: number): void;
    function error(message: DiagnosticMessage, start: number, length: number, ...parameters: string[]): void;
    function error(message: DiagnosticMessage, start?: number, length?: number, ...parameters: string[]): void {
        if (_onError) {
            let errorStart = start || state.startPos;
            let errorLength = length || state.pos - state.startPos + 1;
            let messageText = message.template;
            if (parameters) {
                parameters.forEach((v, i) => messageText = messageText.replace("{" + i.toString() + "}", v));
            }
            const diag: Diagnostic = {
                flag: message.flag,
                code: message.code,
                start: errorStart,
                length: errorLength,
                message: messageText,
            };
            _onError(diag);
        }
    }

    function currentChar(): number {
        return codePointAt(state.text, state.pos);
    }

    // 对于换行符('_'或'\')，需要在文本扫描阶段进行跳过处理
    // 对于正常换行(\r\n)，添加precedingLinebreak的TokenFlag的任务由scan方法执行
    function nextChar(): number {
        state.pos++;
        let ch = currentChar();
        let ahead = state.text.charCodeAt(state.pos + 1);
        // 当前字符是换行符时，检查后一个字符是否是换行符('\r','\n')，如果是的话，视为换行符，否则视为一般符号，后续由scan方法处理
        // 对于Identifier，也是可以使用换行符换行的，换行前后的文本应当被连接成完整的字符
        if (isExpressionOrStringLineBreakCharactor(ch) && isLineBreak(ahead)) {
            state.pos++;
            skipSingleLineFeed();
            return currentChar();
        }
        return ch;
    }

    function lookAheadChar(): number | undefined {
        return lookAhead(nextChar);
    }

    // 扫描字符串类型文本，包含两种类型字符串：
    // 1. 单行字符串： 以双引号开头，以双引号结尾，期间如果需要换行，使用'_'或'\'后换行，但是视为单行字符串。
    // 2. 多行字符串： 以'"!'开头，以'!"'结尾，但是'"!'后需要换行，中间可以自由换行。
    // 需要注意，只有'"!'后换行才视为多行字符串，否则视为单行字符串。
    function scanString(): string {
        // 跳过'"'
        state.pos++;

        let result = "";
        let start = state.pos;

        let multiLineString = false;
        // 检查是否是多行字符串：如果'"!'后是换行符，视为多行字符串
        if (currentChar() === CharacterCodes.exclamation && isLineBreak(state.pos + 1)) {
            state.tokenFlag |= TokenFlags.multiLineString;
            multiLineString = true;
            state.pos++;
            start = state.pos;
        }
        else {
            state.tokenFlag |= TokenFlags.singleLineString;
        }

        while (true) {
            // 如果直到文件结尾也没有找到结束符，视为未终结的字符串
            if (state.pos >= state.end) {
                state.tokenFlag |= TokenFlags.unterminated;
                result += state.text.substring(start, state.pos);
                error(errorMessages.unterminatedStringLiteral, start, state.pos);
                break;
            }
            const ch = state.text.charCodeAt(state.pos);
            // 单行字符串中，两个双引号'""'为转义双引号，视为单个双引号，多行字符串不需要转义。
            if (!multiLineString && ch === CharacterCodes.quotation && state.text.charCodeAt(state.pos + 1) === CharacterCodes.quotation) {
                state.pos += 2;
                result += "\"";
                continue;
            }
            // 检查多行字符的结尾，
            if (multiLineString && ch === CharacterCodes.exclamation && state.text.charCodeAt(state.pos + 1) === CharacterCodes.quotation) {
                state.pos += 2;
                break;
            }
            // 单行字符串的结尾
            if (!multiLineString && ch === CharacterCodes.quotation) {
                result += state.text.substring(start, state.pos);
                state.pos++;
                break;
            }
            // 检查单行字符串中的多行连接符，多行链接需要跳过连接符并链接前后的字符值
            if (!multiLineString && isExpressionOrStringLineBreakCharactor(ch)) {
                let nextChar = state.text.charCodeAt(state.pos + 1);
                // 添加当前行的字符串，并更新start值
                if (isLineBreak(nextChar)) {
                    state.pos++;
                    result += state.text.substring(start, state.pos);
                    skipSingleLineFeed();
                    start = state.pos;
                    continue;
                }
            }
            state.pos++;
        }
        return result;
    }

    function scanHexNumber(): string {
        let valueChars: number[] = [];
        let ch = state.text.charCodeAt(state.pos);
        while (true) {
            if (ch >= CharacterCodes.uppercaseA && ch <= CharacterCodes.uppercaseF) {
                ch += CharacterCodes.lowercaseA - CharacterCodes.uppercaseA;
            }
            else if (isExpressionOrStringLineBreakCharactor(ch)) {
                nextChar();
                continue;
            }
            else if (!isHexNumber(ch)) {
                break;
            }
            valueChars.push(ch);
            ch = nextChar();
        }
        return String.fromCharCode(...valueChars);
    }

    function scanOctalNumber(): string {
        let valueChars: number[] = [];
        let ch = state.text.charCodeAt(state.pos);
        while (true) {
            if (!isOctalNumber(ch)) {
                break;
            }
            valueChars.push(ch);
            ch = nextChar();
        }
        return String.fromCharCode(...valueChars);
    }

    function scanDecimalNumber(): { text: string, isFloat: boolean } {
        let hasDot = false;
        let ch = state.text.charCodeAt(state.pos);
        let valueChars: number[] = [];
        while (true) {
            if (!isDigit(ch)) {
                break;
            }
            valueChars.push(ch);
            ch = nextChar();
            // 对于小数，只读取第一个小数点，小数点出现第二次的话，视为当前数字已经结束
            if (ch === CharacterCodes.dot && isDigit(lookAheadChar()) && !hasDot) {
                valueChars.push(ch);
                ch = nextChar();
                hasDot = true;
            }
        }
        let str = String.fromCharCode(...valueChars);
        return { text: str, isFloat: hasDot };
    }

    function getIdentifierToken(): SyntaxKind.identifier | KeywordTokenKind | DelcareKeywordKind | DmsSectionKeywordKind | MDMKeywordKind {
        const len = state.tokenValue.length;
        if (len >= 2 && len <= 18) {
            const ch = state.tokenValue.charCodeAt(0);
            if ((ch >= CharacterCodes.lowercaseA && ch <= CharacterCodes.lowercaseZ) ||
                (ch >= CharacterCodes.uppercaseA && ch <= CharacterCodes.uppercaseZ)) {
                const keyword = stringToMrsKeyword(state.tokenValue);
                if (keyword) {
                    state.tokenFlag |= TokenFlags.mrsKeyword;
                    return state.token = keyword;
                }
                const declareKeyword = stringToDeclareKeyword(state.tokenValue);
                if (declareKeyword) {
                    state.tokenFlag |= TokenFlags.declareKeyword;
                    return state.token = declareKeyword;
                }
                const dmsSectionKeyword = stringToDmsSectionKeyword(state.tokenValue);
                if (dmsSectionKeyword) {
                    state.tokenFlag |= TokenFlags.dmsKeyword;
                    return state.token = dmsSectionKeyword;
                }
                const metadataKeyword = stringToMetadataKeyword(state.tokenValue);
                if (metadataKeyword) {
                    state.tokenFlag |= TokenFlags.metadataKeyword;
                    return state.token = metadataKeyword;
                }
            }
        }
        return state.token = SyntaxKind.identifier;
    }

    function scanIdentifier(startCharactor: number) {
        let ch = startCharactor;
        let values: number[] = [ ch ];
        if (isIdentifierStart(ch)) {
            ch = nextChar();
            while (true) {
                if (state.pos >= state.end || !isIdentifierPart(ch)) {
                    break;
                }
                // Identifier可以使用换行符('_'或'\')进行多行链接，对于'_'和'\'的判断和链接在nextChar方法中进行，此方法不需要关注
                values.push(ch);
                ch = nextChar();
            }
            state.tokenValue = String.fromCharCode(...values);
            return getIdentifierToken();
        }
        return undefined;
    }

    function speculationHelper<T>(callback: () => T, isLookAhead: boolean): T {
        const saved = saveState();
        const result = callback();
        if (!result || isLookAhead) {
            state = saved;
        }
        return result;
    }

    function lookAhead<T>(callback: () => T): T {
        return speculationHelper(callback, true);
    }

    function tryScan<T>(callback: () => T): T {
        return speculationHelper(callback, false);
    }

    function scanToLinefeedAsString(): SyntaxKind.stringLiteral {
        skipWhitespace();
        let start = state.pos;
        let result = "";
        while (state.pos < state.end) {
            const ch = state.text.charCodeAt(state.pos);
            const ahead = state.text.charCodeAt(state.pos + 1);
            // 中间可以使用'_'或'\'进行换行
            if (isExpressionOrStringLineBreakCharactor(ch) && isLineBreak(ahead)) {
                result += state.text.substring(start, state.pos);
                state.pos++;
                skipSingleLineFeed();
                start = state.pos;
                continue;
            }
            if (isLineBreak(ch)) {
                break;
            }
            state.pos++;
        }
        result += state.text.substring(start, state.pos);
        state.tokenValue = result;
        return SyntaxKind.stringLiteral;
    }

    function scan(): SyntaxKind {

        state.startPos = state.pos;
        state.tokenFlag = TokenFlags.none;

        skipTrivia();

        state.tokenStart = state.pos;
        if (state.pos >= state.end) {
            return state.token = SyntaxKind.eof;
        }

        let ch = currentChar();
        let ahead: number | undefined;
        let current: number;
        const size = charSize(ch);

        switch (ch) {

            case CharacterCodes.openCurlyBrace:
                nextChar();
                return state.token = SyntaxKind.openCurly;
            case CharacterCodes.closeCurlyBrace:
                nextChar();
                return state.token = SyntaxKind.closeCurly;
            case CharacterCodes.openParenthesis:
                nextChar();
                return state.token = SyntaxKind.openParenthesis;
            case CharacterCodes.closeParenthesis:
                nextChar();
                return state.token = SyntaxKind.closeParenthesis;
            case CharacterCodes.openSquareBracket:
                nextChar();
                return state.token = SyntaxKind.openBracket;
            case CharacterCodes.closeSquareBracket:
                nextChar();
                return state.token = SyntaxKind.closeBracket;
            case CharacterCodes.dot:
                if (lookAheadChar() === CharacterCodes.dot) {
                    // 跳过两个字符
                    nextChar();
                    nextChar();
                    if (lookAheadChar() === CharacterCodes.dot && state.context === ScannerContextType.preprocessor) {
                        nextChar();
                        return state.token = SyntaxKind.dotDotDot;
                    }
                    return state.token = SyntaxKind.dotDot;
                }
                nextChar();
                return state.token = SyntaxKind.dot;
            case CharacterCodes.comma:
                nextChar();
                return state.token = SyntaxKind.comma;
            case CharacterCodes.colon:
                nextChar();
                return state.token = SyntaxKind.colon;
            case CharacterCodes.semicolon:
                nextChar();
                return state.token = SyntaxKind.semicolon;
            case CharacterCodes.underscore:
                // 前置检查"_"换行符
                if (isExpressionOrStringLineBreakCharactor(ch) && isLineBreak(codePointAt(state.text, state.pos + 1))) {
                    state.pos++;
                    skipSingleLineFeed();
                    ch = currentChar();
                    return scan();
                }
                const id = scanIdentifier(CharacterCodes.underscore);
                if (id) {
                    return id;
                }
                nextChar();
                return state.token = SyntaxKind.unknown;
            case CharacterCodes.numberSign:
                skipWhitespace();
                nextChar();
                skipWhitespace();
                current = currentChar();
                if (isPreprocessorStart(current)) {
                    // 跳过 # 后的空格, 预处理指令允许 #  define 此类格式
                    ahead = lookAhead(() => {
                        skipWhitespace();
                        return scanIdentifier(currentChar());
                    });
                    if (ahead === SyntaxKind.identifier ||
                        ahead === SyntaxKind.mdmDefineKeyword ||
                        ahead === SyntaxKind.ifKeyword ||
                        ahead === SyntaxKind.elseKeyword ||
                        ahead === SyntaxKind.errorKeyword
                    ) {
                        skipWhitespace();
                        current = currentChar();
                        scanIdentifier(current);
                        const preFlag = stringToPreprocessorKeyword(state.tokenValue);
                        if (preFlag) {
                            return state.token = preFlag;
                        }
                    }
                }
                state.tokenValue = "#";
                error(errorMessages.invalidIdentifierName, state.pos - 1, state.pos, "#");
                return state.token = SyntaxKind.unknown;

            case CharacterCodes.greaterThan:
                ahead = lookAheadChar();
                if (ahead === CharacterCodes.equalsTo) {
                    nextChar();
                    nextChar();
                    return state.token = SyntaxKind.greaterEqual;
                }
                nextChar();
                return state.token = SyntaxKind.greater;
            case CharacterCodes.lessThan:
                ahead = lookAheadChar();
                if (ahead === CharacterCodes.equalsTo) {
                    nextChar();
                    nextChar();
                    return state.token = SyntaxKind.lessEqual;
                } else if (ahead === CharacterCodes.greaterThan) {
                    nextChar();
                    nextChar();
                    return state.token = SyntaxKind.notEqual;
                }
                nextChar();
                return state.token = SyntaxKind.less;
            case CharacterCodes.equalsTo:
                ahead = lookAheadChar();
                if (ahead === CharacterCodes.equalsTo) {
                    nextChar();
                    nextChar();
                    state.tokenFlag |= TokenFlags.ppOperator;
                    return state.token = SyntaxKind.equalEqual;
                }
                else if (ahead === CharacterCodes.asterisk) {
                    nextChar();
                    nextChar();
                    state.tokenFlag |= TokenFlags.operator;
                    return state.token = SyntaxKind.equalAsterisk;
                }
                nextChar();
                state.tokenFlag |= TokenFlags.operator;
                return state.token = SyntaxKind.equal;

            case CharacterCodes.plus:
                nextChar();
                state.tokenFlag |= TokenFlags.operator;
                return state.token = SyntaxKind.plus;
            case CharacterCodes.minus:
                nextChar();
                state.tokenFlag |= TokenFlags.operator;
                return state.token = SyntaxKind.minus;
            case CharacterCodes.slash:
                nextChar();
                state.tokenFlag |= TokenFlags.operator;
                return state.token = SyntaxKind.slash;
            case CharacterCodes.asterisk:
                nextChar();
                state.tokenFlag |= TokenFlags.operator;
                return state.token = SyntaxKind.asterisk;

            case CharacterCodes.verticalBar:
                if (lookAheadChar() === CharacterCodes.verticalBar) {
                    nextChar();
                    nextChar();
                    state.tokenFlag |= TokenFlags.ppOperator;
                    return state.token = SyntaxKind.barbar;
                }
                error(errorMessages.invalidCharactor, state.pos, size, state.text.substring(state.pos, state.pos + size));
                nextChar();
                return state.token = SyntaxKind.unknown;

            case CharacterCodes.quotation:
                state.tokenValue = scanString();
                return state.token = SyntaxKind.stringLiteral;

            case CharacterCodes.exclamation:
                state.tokenFlag |= TokenFlags.ppOperator;
                if (lookAheadChar() === CharacterCodes.equalsTo) {
                    nextChar();
                    nextChar();
                    return state.token = SyntaxKind.exclamationEqual;
                }
                nextChar();
                return state.token = SyntaxKind.exclamation;

            case CharacterCodes.digit0:
            case CharacterCodes.digit1:
            case CharacterCodes.digit2:
            case CharacterCodes.digit3:
            case CharacterCodes.digit4:
            case CharacterCodes.digit5:
            case CharacterCodes.digit6:
            case CharacterCodes.digit7:
            case CharacterCodes.digit8:
            case CharacterCodes.digit9:
                let decimal = scanDecimalNumber();
                state.tokenValue = decimal.text;
                if (decimal.isFloat) {
                    state.tokenFlag |= TokenFlags.float;
                }
                else {
                    state.tokenFlag |= TokenFlags.decimal;
                }
                return state.token = SyntaxKind.numberLiteral;

            case CharacterCodes.ampersand:
                ahead = lookAheadChar();
                if (ahead === CharacterCodes.ampersand) {
                    nextChar();
                    nextChar();
                    state.tokenValue = "&&";
                    state.tokenFlag |= TokenFlags.ppOperator;
                    return state.token = SyntaxKind.ampersandAmpersand;
                }
                else if (ahead === CharacterCodes.uppercaseH || ahead === CharacterCodes.lowercaseH) {
                    // 跳过&H/&h
                    nextChar();
                    nextChar();
                    state.tokenFlag |= TokenFlags.hex;
                    state.tokenValue = scanHexNumber();
                    return state.token = SyntaxKind.numberLiteral;
                }
                else if (ahead === CharacterCodes.uppercaseO || ahead === CharacterCodes.lowercaseO) {
                    // 跳过&O/&o
                    nextChar();
                    nextChar();
                    state.tokenFlag |= TokenFlags.octal;
                    state.tokenValue = scanOctalNumber();
                    return state.token = SyntaxKind.numberLiteral;
                }

                error(errorMessages.invalidCharactor, state.pos, size, state.text.substring(state.pos, state.pos + size));
                nextChar();
                return state.token = SyntaxKind.unknown;

            case CharacterCodes.backslash:
                // \\. 只有在元数据上下文中才有效
                if (state.context === ScannerContextType.metadata &&
                    lookAheadChar() === CharacterCodes.backslash &&
                    state.pos + 2 < state.text.length - 1 &&
                    codePointAt(state.text, state.pos + 2) === CharacterCodes.dot
                ) {
                    state.pos += 3;
                    state.tokenValue = "\\\\.";
                    return state.token = SyntaxKind.backslashBackslashDot;
                }
                state.pos += size;
                return state.token = SyntaxKind.unknown;

            default:
                const identifier = scanIdentifier(ch);
                if (identifier) {
                    return state.token = identifier;
                }

                error(errorMessages.invalidCharactor, state.pos, size, state.text.substring(state.pos, state.pos + size));
                state.pos += size;
                return state.token = SyntaxKind.unknown;
        }
    }

    function scanRange<T>(start: number, length: number, callback: () => T): T {
        const savedState = saveState();
        setText(state.text, start, length);
        const result = callback();
        state = savedState;
        return result;
    }

    function reScanInvalidIdentifier(): SyntaxKind {
        state.pos = state.tokenStart;
        state.tokenFlag = 0;
        const ch = codePointAt(state.text, state.pos);
        const identifier = scanIdentifier(ch);
        if (identifier) {
            return state.token = identifier;
        }
        state.pos += charSize(ch);
        return state.token;
    }

}

/**
 * 将预处理器使用的C语言操作符转换为对应的DS脚本操作符
 * @param token
 * @returns
 */
export function convertPreprocessorOperatorToEquivalentToken(token: SyntaxKind): SyntaxKind {
    switch (token) {
        case SyntaxKind.barbar:                  return SyntaxKind.orKeyword;
        case SyntaxKind.ampersandAmpersand:      return SyntaxKind.andKeyword;
        case SyntaxKind.exclamation:             return SyntaxKind.notKeyword;
        case SyntaxKind.exclamationEqual:        return SyntaxKind.notEqual;
        case SyntaxKind.equalEqual:              return SyntaxKind.equal;
        default:                                 break;
    }
    return token;
}

const unicodeIdentifierStart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1610, 1649, 1747, 1749, 1749, 1765, 1766, 1786, 1788, 1808, 1808, 1810, 1836, 1920, 1957, 2309, 2361, 2365, 2365, 2384, 2384, 2392, 2401, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2524, 2525, 2527, 2529, 2544, 2545, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2649, 2652, 2654, 2654, 2674, 2676, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2749, 2749, 2768, 2768, 2784, 2784, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2877, 2877, 2908, 2909, 2911, 2913, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3168, 3169, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3294, 3294, 3296, 3297, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3424, 3425, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3585, 3632, 3634, 3635, 3648, 3654, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3760, 3762, 3763, 3773, 3773, 3776, 3780, 3782, 3782, 3804, 3805, 3840, 3840, 3904, 3911, 3913, 3946, 3976, 3979, 4096, 4129, 4131, 4135, 4137, 4138, 4176, 4181, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6067, 6176, 6263, 6272, 6312, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8319, 8319, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12329, 12337, 12341, 12344, 12346, 12353, 12436, 12445, 12446, 12449, 12538, 12540, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64285, 64287, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65136, 65138, 65140, 65140, 65142, 65276, 65313, 65338, 65345, 65370, 65382, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ];
const unicodeIdentifierPart = [170, 170, 181, 181, 186, 186, 192, 214, 216, 246, 248, 543, 546, 563, 592, 685, 688, 696, 699, 705, 720, 721, 736, 740, 750, 750, 768, 846, 864, 866, 890, 890, 902, 902, 904, 906, 908, 908, 910, 929, 931, 974, 976, 983, 986, 1011, 1024, 1153, 1155, 1158, 1164, 1220, 1223, 1224, 1227, 1228, 1232, 1269, 1272, 1273, 1329, 1366, 1369, 1369, 1377, 1415, 1425, 1441, 1443, 1465, 1467, 1469, 1471, 1471, 1473, 1474, 1476, 1476, 1488, 1514, 1520, 1522, 1569, 1594, 1600, 1621, 1632, 1641, 1648, 1747, 1749, 1756, 1759, 1768, 1770, 1773, 1776, 1788, 1808, 1836, 1840, 1866, 1920, 1968, 2305, 2307, 2309, 2361, 2364, 2381, 2384, 2388, 2392, 2403, 2406, 2415, 2433, 2435, 2437, 2444, 2447, 2448, 2451, 2472, 2474, 2480, 2482, 2482, 2486, 2489, 2492, 2492, 2494, 2500, 2503, 2504, 2507, 2509, 2519, 2519, 2524, 2525, 2527, 2531, 2534, 2545, 2562, 2562, 2565, 2570, 2575, 2576, 2579, 2600, 2602, 2608, 2610, 2611, 2613, 2614, 2616, 2617, 2620, 2620, 2622, 2626, 2631, 2632, 2635, 2637, 2649, 2652, 2654, 2654, 2662, 2676, 2689, 2691, 2693, 2699, 2701, 2701, 2703, 2705, 2707, 2728, 2730, 2736, 2738, 2739, 2741, 2745, 2748, 2757, 2759, 2761, 2763, 2765, 2768, 2768, 2784, 2784, 2790, 2799, 2817, 2819, 2821, 2828, 2831, 2832, 2835, 2856, 2858, 2864, 2866, 2867, 2870, 2873, 2876, 2883, 2887, 2888, 2891, 2893, 2902, 2903, 2908, 2909, 2911, 2913, 2918, 2927, 2946, 2947, 2949, 2954, 2958, 2960, 2962, 2965, 2969, 2970, 2972, 2972, 2974, 2975, 2979, 2980, 2984, 2986, 2990, 2997, 2999, 3001, 3006, 3010, 3014, 3016, 3018, 3021, 3031, 3031, 3047, 3055, 3073, 3075, 3077, 3084, 3086, 3088, 3090, 3112, 3114, 3123, 3125, 3129, 3134, 3140, 3142, 3144, 3146, 3149, 3157, 3158, 3168, 3169, 3174, 3183, 3202, 3203, 3205, 3212, 3214, 3216, 3218, 3240, 3242, 3251, 3253, 3257, 3262, 3268, 3270, 3272, 3274, 3277, 3285, 3286, 3294, 3294, 3296, 3297, 3302, 3311, 3330, 3331, 3333, 3340, 3342, 3344, 3346, 3368, 3370, 3385, 3390, 3395, 3398, 3400, 3402, 3405, 3415, 3415, 3424, 3425, 3430, 3439, 3458, 3459, 3461, 3478, 3482, 3505, 3507, 3515, 3517, 3517, 3520, 3526, 3530, 3530, 3535, 3540, 3542, 3542, 3544, 3551, 3570, 3571, 3585, 3642, 3648, 3662, 3664, 3673, 3713, 3714, 3716, 3716, 3719, 3720, 3722, 3722, 3725, 3725, 3732, 3735, 3737, 3743, 3745, 3747, 3749, 3749, 3751, 3751, 3754, 3755, 3757, 3769, 3771, 3773, 3776, 3780, 3782, 3782, 3784, 3789, 3792, 3801, 3804, 3805, 3840, 3840, 3864, 3865, 3872, 3881, 3893, 3893, 3895, 3895, 3897, 3897, 3902, 3911, 3913, 3946, 3953, 3972, 3974, 3979, 3984, 3991, 3993, 4028, 4038, 4038, 4096, 4129, 4131, 4135, 4137, 4138, 4140, 4146, 4150, 4153, 4160, 4169, 4176, 4185, 4256, 4293, 4304, 4342, 4352, 4441, 4447, 4514, 4520, 4601, 4608, 4614, 4616, 4678, 4680, 4680, 4682, 4685, 4688, 4694, 4696, 4696, 4698, 4701, 4704, 4742, 4744, 4744, 4746, 4749, 4752, 4782, 4784, 4784, 4786, 4789, 4792, 4798, 4800, 4800, 4802, 4805, 4808, 4814, 4816, 4822, 4824, 4846, 4848, 4878, 4880, 4880, 4882, 4885, 4888, 4894, 4896, 4934, 4936, 4954, 4969, 4977, 5024, 5108, 5121, 5740, 5743, 5750, 5761, 5786, 5792, 5866, 6016, 6099, 6112, 6121, 6160, 6169, 6176, 6263, 6272, 6313, 7680, 7835, 7840, 7929, 7936, 7957, 7960, 7965, 7968, 8005, 8008, 8013, 8016, 8023, 8025, 8025, 8027, 8027, 8029, 8029, 8031, 8061, 8064, 8116, 8118, 8124, 8126, 8126, 8130, 8132, 8134, 8140, 8144, 8147, 8150, 8155, 8160, 8172, 8178, 8180, 8182, 8188, 8255, 8256, 8319, 8319, 8400, 8412, 8417, 8417, 8450, 8450, 8455, 8455, 8458, 8467, 8469, 8469, 8473, 8477, 8484, 8484, 8486, 8486, 8488, 8488, 8490, 8493, 8495, 8497, 8499, 8505, 8544, 8579, 12293, 12295, 12321, 12335, 12337, 12341, 12344, 12346, 12353, 12436, 12441, 12442, 12445, 12446, 12449, 12542, 12549, 12588, 12593, 12686, 12704, 12727, 13312, 19893, 19968, 40869, 40960, 42124, 44032, 55203, 63744, 64045, 64256, 64262, 64275, 64279, 64285, 64296, 64298, 64310, 64312, 64316, 64318, 64318, 64320, 64321, 64323, 64324, 64326, 64433, 64467, 64829, 64848, 64911, 64914, 64967, 65008, 65019, 65056, 65059, 65075, 65076, 65101, 65103, 65136, 65138, 65140, 65140, 65142, 65276, 65296, 65305, 65313, 65338, 65343, 65343, 65345, 65370, 65381, 65470, 65474, 65479, 65482, 65487, 65490, 65495, 65498, 65500, ];

function lookupInUnicodeMap(code: number, map: readonly number[]): boolean {

    if (code < map[0]) {
        return false;
    }

    // Binary search
    let lo = 0;
    let hi = map.length;
    let mid: number;

    while (lo + 1 < hi) {
        mid = lo + (hi - lo) / 2;
        mid -= mid % 2;
        if (map[mid] <= code && code <= map[mid + 1]) {
            return true;
        }

        if (code < map[mid]) {
            hi = mid;
        }
        else {
            lo = mid + 2;
        }
    }

    return false;
}

export function isUnicodeIdentifierStart(code: number): boolean {
    return lookupInUnicodeMap(code, unicodeIdentifierStart);
}

export function isUnicodeIdentifierPart(code: number): boolean {
    return lookupInUnicodeMap(code, unicodeIdentifierPart);
}

export function isIdentifierStart(code: number | undefined): boolean {
    return code !== undefined && (
        (code >= CharacterCodes.lowercaseA && code <= CharacterCodes.lowercaseZ) ||
        (code >= CharacterCodes.uppercaseA && code <= CharacterCodes.uppercaseZ) ||
        code === CharacterCodes.underscore || code === CharacterCodes.dollarSign ||
        code === CharacterCodes.atSign || code === CharacterCodes.numberSign || isUnicodeIdentifierStart(code));
}

export function isIdentifierPart(code: number | undefined): boolean {
    return code !== undefined && (isIdentifierStart(code) || isDigit(code));
}

export function isPreprocessorStart(code: number): boolean {
    return code === CharacterCodes.lowercaseI ||
        code === CharacterCodes.lowercaseD ||
        code === CharacterCodes.lowercaseE ||
        code === CharacterCodes.lowercaseL ||
        code === CharacterCodes.lowercaseU;
}

export enum CharacterCodes {
    backSpace = 8,
    tab = 9,                //  '\t'
    lineFeed = 10,          //  '\n'
    carriageReturn = 13,    //  '\r'
    shiftOut = 14,
    space = 32,
    exclamation = 33,        //  '!'
    quotation = 34,          //  '"'
    numberSign = 35,         //  '#'
    dollarSign = 36,         //  '$'
    percentSign = 37,        //  '%'
    ampersand = 38,          //  '&'
    apostrophe = 39,         //  '''
    openParenthesis = 40,    //  '('
    closeParenthesis = 41,   //  ')'
    asterisk = 42,           //  '*'
    plus = 43,               //  '+'
    comma = 44,              //  ','
    minus = 45,              //  '-'
    dot = 46,                //  '.'
    slash = 47,              //  '/'
    digit0 = 48,             //  '0'
    digit1 = 49,             //  '1'
    digit2 = 50,             //  '2'
    digit3 = 51,             //  '3'
    digit4 = 52,             //  '4'
    digit5 = 53,             //  '5'
    digit6 = 54,             //  '6'
    digit7 = 55,             //  '7'
    digit8 = 56,             //  '8'
    digit9 = 57,             //  '9'
    colon = 58,              //  ':'
    semicolon = 59,          //  ';'
    lessThan = 60,           //  '<'
    equalsTo = 61,           //  '='
    greaterThan = 62,        //  '>'
    questionMark = 63,       //  '?'
    atSign = 64,             //  '@'
    uppercaseA = 65,         //  'A'
    uppercaseB = 66,         //  'B'
    uppercaseC = 67,         //  'C'
    uppercaseD = 68,         //  'D'
    uppercaseE = 69,         //  'E'
    uppercaseF = 70,         //  'F'
    uppercaseG = 71,         //  'G'
    uppercaseH = 72,         //  'H'
    uppercaseI = 73,         //  'I'
    uppercaseJ = 74,         //  'J'
    uppercaseK = 75,         //  'K'
    uppercaseL = 76,         //  'L'
    uppercaseM = 77,         //  'M'
    uppercaseN = 78,         //  'N'
    uppercaseO = 79,         //  'O'
    uppercaseP = 80,         //  'P'
    uppercaseQ = 81,         //  'Q'
    uppercaseR = 82,         //  'R'
    uppercaseS = 83,         //  'S'
    uppercaseT = 84,         //  'T'
    uppercaseU = 85,         //  'U'
    uppercaseV = 86,         //  'V'
    uppercaseW = 87,         //  'W'
    uppercaseX = 88,         //  'X'
    uppercaseY = 89,         //  'Y'
    uppercaseZ = 90,         //  'Z'
    openSquareBracket = 91,  //  '['
    backslash = 92,          //  '\'
    closeSquareBracket = 93, //  ']'
    caret = 94,              //  '^'
    underscore = 95,         //  '_'
    graveAccent = 96,        //  '`'
    lowercaseA = 97,         //  'a'
    lowercaseB = 98,         //  'b'
    lowercaseC = 99,         //  'c'
    lowercaseD = 100,        //  'd'
    lowercaseE = 101,        //  'e'
    lowercaseF = 102,        //  'f'
    lowercaseG = 103,        //  'g'
    lowercaseH = 104,        //  'h'
    lowercaseI = 105,        //  'i'
    lowercaseJ = 106,        //  'j'
    lowercaseK = 107,        //  'k'
    lowercaseL = 108,        //  'l'
    lowercaseM = 109,        //  'm'
    lowercaseN = 110,        //  'n'
    lowercaseO = 111,        //  'o'
    lowercaseP = 112,        //  'p'
    lowercaseQ = 113,        //  'q'
    lowercaseR = 114,        //  'r'
    lowercaseS = 115,        //  's'
    lowercaseT = 116,        //  't'
    lowercaseU = 117,        //  'u'
    lowercaseV = 118,        //  'v'
    lowercaseW = 119,        //  'w'
    lowercaseX = 120,        //  'x'
    lowercaseY = 121,        //  'y'
    lowercaseZ = 122,        //  'z'
    openCurlyBrace = 123,    //  '{'
    verticalBar = 124,       //  '|'
    closeCurlyBrace = 125,   //  '}'
    tilde = 126,             //  '~'
    nonBreakingSpace = 160,
    oghamSpaceMark = 5760,   // ' '
    lineSeparator = 8232,
    paragraphSeparator = 8233,
    maxAsciiCharacter = 127,
}

function isDigit(code: number | undefined) {
    return code !== undefined && (code >= CharacterCodes.digit0 && code <= CharacterCodes.digit9);
}

function isHexNumber(code: number | undefined): boolean {
    return code !== undefined && (isDigit(code) || (code >= CharacterCodes.uppercaseA && code <= CharacterCodes.uppercaseF) || (code >= CharacterCodes.lowercaseA && code <= CharacterCodes.lowercaseF));
}

function isOctalNumber(code: number | undefined): boolean {
    return code !== undefined && (code >= CharacterCodes.digit0 && code <= CharacterCodes.digit7);
}

function isLineBreak(code: number | undefined): boolean {
    return code !== undefined && (code === CharacterCodes.lineFeed || code === CharacterCodes.carriageReturn || code === CharacterCodes.lineSeparator || code === CharacterCodes.paragraphSeparator);
}

function isCodePoint(code: number): boolean {
    return code <= 0x10FFFF;
}

function isExpressionOrStringLineBreakCharactor(code: number | undefined): boolean {
    return code !== undefined && (code === CharacterCodes.underscore || code === CharacterCodes.backslash);
}

/* @internal */
const codePointAt: (s: string, i: number) => number = (String.prototype as any).codePointAt ? (s, i) => (s as any).codePointAt(i) : function codePointAt(str, i): number {
    // from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/codePointAt
    const size = str.length;
    // Account for out-of-bounds indices:
    if (i < 0 || i >= size) {
        return undefined!; // String.codePointAt returns `undefined` for OOB indexes
    }
    // Get the first code unit
    const first = str.charCodeAt(i);
    // check if it’s the start of a surrogate pair
    if (first >= 0xD800 && first <= 0xDBFF && size > i + 1) { // high surrogate and there is a next code unit
        const second = str.charCodeAt(i + 1);
        if (second >= 0xDC00 && second <= 0xDFFF) { // low surrogate
            // https://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
            return (first - 0xD800) * 0x400 + second - 0xDC00 + 0x10000;
        }
    }
    return first;
};

/* @internal */
function charSize(ch: number) {
    if (ch >= 0x10000) {
        return 2;
    }
    return 1;
}

function isWhiteSpaceSingleLine(code: number): boolean {
    return code === CharacterCodes.space ||
        code === CharacterCodes.tab   ||
        code === CharacterCodes.oghamSpaceMark;
}

function isWhiteSpaceLike(code: number): boolean {
    return isWhiteSpaceSingleLine(code) || isLineBreak(code);
}

export function computeLineStarts(text: string): number[] {
    const result: number[] = [];
    let pos = 0;
    let lineStart = 0;
    while (pos < text.length) {
        const ch = text.charCodeAt(pos);
        pos++;
        switch (ch) {
            case CharacterCodes.carriageReturn:
                if (text.charCodeAt(pos) === CharacterCodes.lineFeed) {
                    pos++;
                }

            case CharacterCodes.lineFeed:
                result.push(lineStart);
                lineStart = pos;
                break;

            default:
                if (ch > CharacterCodes.maxAsciiCharacter && isLineBreak(ch)) {
                    result.push(lineStart);
                    lineStart = pos;
                }
                break;
        }
    }
    result.push(lineStart);
    return result;
}

export function computeLineOfPosition(lineStarts: readonly number[], position: number, offset?: number) {
    let lineNumber = binarySearch(lineStarts, position, compareNumberValue, offset);
    if (lineNumber < 0) {
        lineNumber = ~lineNumber - 1;
    }
    return lineNumber;
}

export function computeLineAndCharacterOfLastLineEndPosition(lineStarts: readonly number[], position: number): TextPosition {
    let line = computeLineOfPosition(lineStarts, position);
    let character = position - lineStarts[line];
    if (lineStarts[line] > 0) {
        line--;
        character = lineStarts[line + 1] - 1 - lineStarts[line];
    }
    return {
        line,
        character,
    };
}

export function computeLineAndCharacterOfPosition(lineStarts: readonly number[], position: number): TextPosition {
    const line = computeLineOfPosition(lineStarts, position);
    return {
        line,
        character: position - lineStarts[line]
    };
}

export function computePositionOfLineAndCharacter(lineStarts: readonly number[], line: number, character: number): number {
    if (line < 0 || line >= lineStarts.length) {
        line = line < 0 ? 0 : lineStarts.length - 1;
    }
    return lineStarts[line] + character;
}

export function isLigalIdentifier(str: string): boolean {
    if (str.length === 0) {
        return false;
    }

    if (str.length === 1) {
        return isIdentifierStart(codePointAt(str, 0));
    }

    if (isIdentifierStart(codePointAt(str, 0))) {
        for (let i = 1; i < str.length; i++) {
            if (!isIdentifierPart(codePointAt(str, i))) {
                return false;
            }
        }
        return true;
    }

    return false;
}
